import get_page
from bs4 import BeautifulSoup
from multiprocessing import Pool
import json
def get_ip_from_xicidaili(page):
    data_list=[]
    #url : http://www.xicidaili.com/nn/1
    url='http://www.xicidaili.com/nn/'+str(page)
    resp=get_page.get_page(url)
    soup=BeautifulSoup(resp.text,'lxml')
    ip_list=soup.find(id='ip_list').find_all('tr')
    count=0
    for each in ip_list:
        if(count==0):
            pass
        else:
            td_list=each.find_all('td')
            ip=td_list[1].get_text()+':'+td_list[2].get_text()
            kind=td_list[5].get_text()
            temp={}
            if(kind=='HTTP'):
                temp['HTTP']=ip
            elif(kind=='HTTPS'):
                temp['HTTPS']=ip
            else:
                pass

            data_list.append(temp)
            # print('{} : {}'.format(kind,ip))
        count+=1
    print('获取 {} 个代理 from page {} '.format(count,str(page)))
    return data_list

def main_xicidaili():
    data=[]
    pool=Pool()
    results=pool.map(get_ip_from_xicidaili,[i for i in range(1,2)])
    for each in results:
        for each_dict in each:
            data.append(each_dict)
    with open('{}.json'.format('xicidaili111'), 'w', encoding='utf-8')as file:
        json.dump(data, file, ensure_ascii=False)

    print('ok')

if __name__ == '__main__':
    main_xicidaili()
